#include "dictionary.h"

dictionary::dictionary(void) {
	dictTable = NULL;
	htbDict = NULL;
	totalEntry = 0;
	limSizeDict = 0;
}

dictionary::dictionary( const string &inputFilename )
{
	ifstream inputFile(inputFilename.c_str());
	if (!inputFile) {
		cerr << "Cannot open input file " << inputFilename << ". Exiting..." << endl;
		exit(1);
	}

	char buffer[30];
	inputFile >> setw(30) >> buffer;
	inputFile.close();

	if (strncmp(buffer, "DICT_ENTRY_FILTER", 17) == 0)
		load(inputFilename);
	else
		generate(inputFilename);
}

dictionary::~dictionary(void) {
	delete htbDict;
	delete[] dictTable;

	delete st;
}

void dictionary::tokenize( const string& str, const char& delimiter, vector<string>& tokens ) {
	// Skip delimiters at beginning. Find first "non-delimiter".
	string::size_type lastPos = str.find_first_not_of(delimiter, 0);
	// Find next instance of the delimiter.
	string::size_type pos = str.find_first_of(delimiter, lastPos);

	while (string::npos != pos || string::npos != lastPos) {
		// Add the token from lastPos to pos.
		tokens.push_back(str.substr(lastPos, pos - lastPos));

		// Find the start of the next token.
		lastPos = str.find_first_not_of(delimiter, pos);

		// Find the end of the token.
		pos = str.find_first_of(delimiter, lastPos);
	}
}

void dictionary::grow() {
	delete htbDict;

	cerr << "D+\b";
	unsigned int newlim = (unsigned int) (limSizeDict * GROWTH_STEP_1);
	dict_entry *newDictTable = new dict_entry[newlim];

	memcpy(newDictTable, dictTable, sizeof(dict_entry) * limSizeDict);

	delete[] dictTable;
	dictTable = newDictTable;

	htbDict = new htable<char*>((size_t) (newlim / DICTIONARY_LOAD_FACTOR_1));
	for (unsigned int i = 0; i < limSizeDict; ++i) {
		htbDict->insert((char*) &dictTable[i].word);
	}

	limSizeDict = newlim;
}

void dictionary::save( const string &outputFilename ) {
	ofstream outputFile(outputFilename.c_str());
	if (!outputFile) {
		cerr << "Cannot open output file " << outputFilename << ". Exiting..." << endl;
		exit(1);
	}

	// ---
	outputFile << "DICT_ENTRY\t" << totalEntry << endl;
	for (unsigned int i = 0; i < totalEntry; ++i) {
		outputFile << dictTable[i].word << "\t" << dictTable[i].freq << endl;
	}
}

void dictionary::save( const string &outputFilename, const int &thresholdFreq )
{
	ofstream outputFile(outputFilename.c_str());
	if (!outputFile) {
		cerr << "Cannot open output file " << outputFilename << ". Exiting..." << endl;
		exit(1);
	}

	// ---
	outputFile << "DICT_ENTRY_FILTER\n";
	for (unsigned int i = 0; i < totalEntry; ++i){
		if (dictTable[i].freq >= (unsigned long)thresholdFreq) {
			outputFile << dictTable[i].word << "\t" << dictTable[i].freq << endl;
			//outputFile << dictTable[i].word << endl;
		}
	}
}

void dictionary::stat( const string &outputFilename ) {
	ofstream outputFile(outputFilename.c_str());
	if (!outputFile) {
		cerr << "Cannot open output file " << outputFilename << ". Exiting..." << endl;
		exit(1);
	}

	// ---
	outputFile << "DICT_STAT\t" << totalEntry << endl;
	map<unsigned long long, unsigned int> mStat;
	unsigned long long numberofWords = 0;
	for (unsigned int i = 0; i < totalEntry; ++i) {
		mStat[dictTable[i].freq] += 1;
		numberofWords += dictTable[i].freq;
	}
	outputFile << numberofWords << endl;

	map<unsigned long long, unsigned int>::iterator itStat;
	for (itStat = mStat.begin(); itStat != mStat.end(); ++itStat) {
		outputFile << itStat->first << "\t" << itStat->second << endl;
	}
}

void dictionary::generate( const string &inputFilename ) {
	ifstream inputFile(inputFilename.c_str());
	if (!inputFile) {
		cerr << "Cannot open input file " << inputFilename << ". Exiting..." << endl;
		exit(1);
	}

	// ---
	totalEntry = 0;
	limSizeDict = 5000;
	dictTable = new dict_entry[limSizeDict];
	htbDict = new htable<char*>((size_t) (limSizeDict / DICTIONARY_LOAD_FACTOR_1));

	st = new strstack(limSizeDict * 10);

	dict_entry *ptrDict;
	char *pToken;

	clock_t clock1, clock2;
	// ---

	clock1 = clock();
	cerr << "Reading input file and creating dictionary." << endl;
	string line;
	unsigned int curr = 0;
	//unsigned int filterIndex = 0;
	// Go through all lines in the input file.
	while (!inputFile.eof()) {
		getline(inputFile, line);
		if (line != "") {
			++curr;
			if (curr % 20000 == 0) {
				cerr << ".";
				if (curr % 100000 == 0) {
					clock2 = clock();
					double partialTime = (((double)(clock2 - clock1)) / ((double)(CLOCKS_PER_SEC)));
					cerr << " processed " << curr << " lines (" << partialTime << " sec) " << endl;
				}
			}

			// Tokenize the line.
			vector<string> tokens;
			tokenize(line, ' ', tokens);
			// Go through all tokens and increment frequency.
			for (unsigned int i = 0; i < tokens.size(); ++i) {
				// Word
				pToken = (char *)tokens[i].c_str();
				ptrDict = (dict_entry *)htbDict->find((char*) &pToken);
				if (ptrDict != NULL) {
					dictTable[ptrDict->code].freq += 1;
				} else {
					dictTable[totalEntry].word = st->push((char *)pToken);
					htbDict->insert((char *)&dictTable[totalEntry].word);
					dictTable[totalEntry].code = totalEntry;
					dictTable[totalEntry].freq = 1;
					++totalEntry;
					// Check number of token
					if (totalEntry == limSizeDict) {
						grow();
					}
				}
			}
			tokens.clear();
		}
		line.clear();
	}
	clock2 = clock();
	double readInputTime = (((double)(clock2 - clock1)) / ((double)(CLOCKS_PER_SEC)));
	cerr << "Done (" << readInputTime << " sec)" << endl << endl;
}

void dictionary::load( const string &inputFilename ) {
	ifstream inputFile(inputFilename.c_str());
	if (!inputFile) {
		cerr << "Cannot open input file " << inputFilename << ". Exiting..." << endl;
		exit(1);
	}

	// ---
	totalEntry = 0;
	limSizeDict = 5000;
	dictTable = new dict_entry[limSizeDict];
	htbDict = new htable<char*>((size_t) (limSizeDict / DICTIONARY_LOAD_FACTOR_1));

	st = new strstack(limSizeDict * 10);

	dict_entry *ptrDict;
	char *pToken;

	clock_t clock1, clock2;
	// ---

	clock1 = clock();
	cerr << "Loading dictionary." << endl;
	string line;
	unsigned int curr = 0;
	//unsigned int filterIndex = 0;
	// Go through all lines in the input file.
	getline(inputFile, line); // Skip header-line
	while (!inputFile.eof()) {
		getline(inputFile, line);
		if (line != "") {
			++curr;
			if (curr % 5000 == 0) {
				cerr << ".";
				if (curr % 20000 == 0) {
					clock2 = clock();
					double partialTime = (((double)(clock2 - clock1)) / ((double)(CLOCKS_PER_SEC)));
					cerr << " processed " << curr << " lines (" << partialTime << " sec) " << endl;
				}
			}

			// Tokenize the line.
			vector<string> tokens;
			tokenize(line, '\t', tokens);
			if (tokens.size() == 2) {
				pToken = (char *)tokens[0].c_str();
				ptrDict = (dict_entry *)htbDict->find((char*) &pToken);
				if (ptrDict != NULL) {
#ifdef WIN32
					dictTable[ptrDict->code].freq += _strtoui64(tokens[1].c_str(), NULL, 0);
#else
					dictTable[ptrDict->code].freq += strtoull(tokens[1].c_str(), NULL, 0);
#endif
				} else {
					dictTable[totalEntry].word = st->push((char *)pToken);
					htbDict->insert((char *)&dictTable[totalEntry].word);
					dictTable[totalEntry].code = totalEntry;
#ifdef WIN32
					dictTable[totalEntry].freq = _strtoui64(tokens[1].c_str(), NULL, 0);
#else
					dictTable[totalEntry].freq = strtoull(tokens[1].c_str(), NULL, 0);
#endif
					++totalEntry;
					// Check number of token
					if (totalEntry == limSizeDict) {
						grow();
					}
				}
			} else {
				cerr << "Reading dictionary entry error\nLine: " << curr << endl;
			}
			tokens.clear();
		}
		line.clear();
	}
	clock2 = clock();
	double readInputTime = (((double)(clock2 - clock1)) / ((double)(CLOCKS_PER_SEC)));
	cerr << "Done (" << readInputTime << " sec)" << endl << endl;
}

bool dictionary::findEntry( const string &entry ) {
	dict_entry *ptrDict = NULL;
	char *pToken = (char *)entry.c_str();
	ptrDict = (dict_entry *)htbDict->find((char*) &pToken);
	return ptrDict != NULL ? true : false;
}

unsigned int dictionary::size() {
	return totalEntry;
}

unsigned int* dictionary::encode( const string &entry ) const{
	dict_entry *ptrDict = NULL;
	char *pToken = (char *)entry.c_str();
	ptrDict = (dict_entry *)htbDict->find((char*) &pToken);
	return ptrDict != NULL ? &(ptrDict->code) : NULL;
}

unsigned int* dictionary::encode( char* entry ) const {
	dict_entry *ptrDict = NULL;
	ptrDict = (dict_entry *)htbDict->find((char*) &entry);
	return ptrDict != NULL ? &(ptrDict->code) : NULL;
}

char* dictionary::operator[]( unsigned int index ) const{
	if (index < totalEntry && index >= 0) {
		return (char*)dictTable[index].word;
	} else {
		return NULL;
	}
}
